#!/usr/bin/env python
# coding=utf-8
'''
北京青年旅行网
国内旅游：http://www.cytsbj.com/intour/list.php?id=15   id[5-31]
出境线路：http://www.ctsfj.com/line/list.asp?id=2       id[1-16]
('周边线路','跟团游','http://www.cytsbj.com/zbtour/zbtour.php', )
('海岛游线路','跟团游','http://www.cytsbj.com/islandtour/islandtour.php', )
('北京游线路','跟团游','http://www.cytsbj.com/bjtour/bjtour.php', )

大约400条  没有抓取港澳线路（因信息不全）。
'''
import os, sys, re, traceback, logging
import urllib
import time
import datetime
import getopt
import urllib2
from Route import *

vacation_routenum = re.compile(r'''<a\s*href=".id=[0-9]+&page=([0-9]+)"\s*class="next">''', re.X|re.S)
vacation_routeurl = re.compile(r'''<a\s*href="../(tour/[0-9]+?.php)"[\S\s]+?<b>[\S\s]+?</b>\s*</a>''', re.X|re.S)
#周边的num
vacation_zhou_routenum = re.compile(r'''<a\s*href=".page=([0-9]+)"\s*class="next">''', re.X|re.S)
vacation_zhou_routeurl = re.compile(r'''<a\s*href="../(tour/[0-9]+?.php)"[\S\s]+?<b>[\S\s]+?</b>''', re.X|re.S)


re_h=re.compile('<[\S\s]*>') #取出信息中夹杂的html标签

http_url = "http://www.cytsbj.com/"  #北京青年旅行社
route_telphone = "968877"

def do_crawl():
    try:
        server_url_data = [
                ('国内旅游','跟团游','http://www.cytsbj.com/intour/list.php?id=', ),
                ('出境线路','跟团游','http://www.cytsbj.com/outtour/list.php?id=', ),
                ('周边线路','跟团游','http://www.cytsbj.com/zbtour/zbtour.php', ),
                ('海岛游线路','跟团游','http://www.cytsbj.com/islandtour/islandtour.php', ),
                ('北京游线路','跟团游','http://www.cytsbj.com/bjtour/bjtour.php', ),
        ]
        #
        #
        count = 0
        for route_te,routetype,server_url in server_url_data:
            if route_te.find('国内')>-1 and routetype.find('跟团游')>-1:
                for i in range(5,32):
                    page_url = server_url+str(i)
                    try:
                        html = urllib2.urlopen(page_url).read()
                        vacation_route_num = vacation_routenum.findall(html)
                        if vacation_route_num:
                            print "page:"+vacation_route_num[3]
                            for i in range(int(vacation_route_num[3].strip())):
                                page_url2 = page_url+"&page="+str(i+1)
                                print "page_url:"+page_url2
                                '''获取route_url'''
                                vacation_route_url=""
                                try:
                                    html = urllib2.urlopen(page_url2).read()
                                    #html = getUTF(html)
                                    vacation_route_url = vacation_routeurl.findall(html)
                                    for i in vacation_route_url:
                                        print "route_url:"+http_url+i
                                        route_url = http_url+i
                                        getRoutes(route_url,routetype)
                                except Exception,e:
                                    print traceback.format_exc(sys.exc_info())
                    except Exception,e:
                        print traceback.format_exc(sys.exc_info())
            
            if route_te.find('出境')>-1 and routetype.find('跟团游')>-1:
                for i in range(1,17):
                    page_url = server_url+str(i)
                    try:
                        html = urllib2.urlopen(page_url).read()
                        vacation_route_num = vacation_routenum.findall(html)
                        if vacation_route_num:
                            print "page:"+vacation_route_num[3]
                            for i in range(int(vacation_route_num[3].strip())):
                                page_url2 = page_url+"&page="+str(i+1)
                                print "page_url:"+page_url2
                                '''获取route_url'''
                                vacation_route_url=""
                                try:
                                    html = urllib2.urlopen(page_url2).read()
                                    #html = getUTF(html)
                                    vacation_route_url = vacation_routeurl.findall(html)
                                    for i in vacation_route_url:
                                        print "route_url:"+http_url+i
                                        route_url = http_url+i
                                        getRoutes(route_url,routetype)
                                except Exception,e:
                                    print traceback.format_exc(sys.exc_info())
                    except Exception,e:
                        print traceback.format_exc(sys.exc_info())
            
            if route_te.find('周边')>-1 and routetype.find('跟团游')>-1:
                    page_url = server_url
                    print page_url
                    try:
                        html = urllib2.urlopen(page_url).read()
                        html = getUTF(html)
                        vacation_route_num = vacation_zhou_routenum.findall(html)
                        if vacation_route_num:
                            print "page:"+vacation_route_num[3]
                            for i in range(int(vacation_route_num[3].strip())):
                                page_url2 = page_url+"?page="+str(i+1)
                                print "page_url:"+page_url2
                                '''获取route_url'''
                                vacation_route_url=""
                                try:
                                    html = urllib2.urlopen(page_url2).read()
                                    #html = getUTF(html)
                                    vacation_route_url = vacation_zhou_routeurl.findall(html)
                                    for i in vacation_route_url:
                                        print "route_url:"+http_url+i
                                        route_url = http_url+i
                                        getRoutes(route_url,routetype)
                                except Exception,e:
                                    print traceback.format_exc(sys.exc_info())
                    except Exception,e:
                        print traceback.format_exc(sys.exc_info())
            
            if route_te.find('海岛')>-1 and routetype.find('跟团游')>-1:
                    page_url = server_url
                    print page_url
                    try:
                        html = urllib2.urlopen(page_url).read()
                        html = getUTF(html)
                        vacation_route_num = vacation_zhou_routenum.findall(html)
                        if vacation_route_num:
                            print "page:"+vacation_route_num[3]
                            for i in range(int(vacation_route_num[3].strip())):
                                page_url2 = page_url+"?page="+str(i+1)
                                print "page_url:"+page_url2
                                '''获取route_url'''
                                vacation_route_url=""
                                try:
                                    html = urllib2.urlopen(page_url2).read()
                                    #html = getUTF(html)
                                    vacation_route_url = vacation_zhou_routeurl.findall(html)
                                    for i in vacation_route_url:
                                        print "route_url:"+http_url+i
                                        route_url = http_url+i
                                        getRoutes(route_url,routetype)
                                except Exception,e:
                                    print traceback.format_exc(sys.exc_info())
                    except Exception,e:
                        print traceback.format_exc(sys.exc_info())
            
            if route_te.find('北京游')>-1 and routetype.find('跟团游')>-1:
                    page_url = server_url
                    print page_url
                    try:
                        html = urllib2.urlopen(page_url).read()
                        html = getUTF(html)
                        vacation_route_num = vacation_zhou_routenum.findall(html)
                        if vacation_route_num:
                            print "page:"+vacation_route_num[3]
                            for i in range(int(vacation_route_num[3].strip())):
                                page_url2 = page_url+"?page="+str(i+1)
                                print "page_url:"+page_url2
                                '''获取route_url'''
                                vacation_route_url=""
                                try:
                                    html = urllib2.urlopen(page_url2).read()
                                    #html = getUTF(html)
                                    vacation_route_url = vacation_zhou_routeurl.findall(html)
                                    for i in vacation_route_url:
                                        print "route_url:"+http_url+i
                                        route_url = http_url+i
                                        getRoutes(route_url,routetype)
                                except Exception,e:
                                    print traceback.format_exc(sys.exc_info())
                    except Exception,e:
                        print traceback.format_exc(sys.exc_info())
            
    except Exception,e:
        print traceback.format_exc(sys.exc_info())        

route_title_re = re.compile(r'''id=.xl.>\s*<div\s*class=.hd.>\s*<h1>([\S\s]+)</h1><span>([\S\s]+?)</span>\s*</div>\s*<div\s*class="bd">''', re.X|re.S)
route_detail_re = re.compile(r'''<div\s*class=.nr1.>[\S\s]+?>([\S\s]+?)</div>''', re.X|re.S)
route_planes_re = re.compile(r'''<div\s*class=.hd3.>\s*<h3>([\S\s]+?)</h3>\s*</div>\s*<div\s*class=.bd3.>([\S\s]+?)<span>''', re.X|re.S)
route_sights_re = re.compile(r'''<td\s*width="50%"\s*class="font3">([\S\s]+?)</td>''', re.X|re.S)
route_dates_re = re.compile(r'''onclick=.goto_booking[\S\s]+?(\d{4}\-\d{2}\-\d{2})[\S\s]+?\)''', re.X|re.S)
route_trip_all_re = re.compile(r'''<div\s*class="box4">([\S\s]+?)</div>''', re.X|re.S)
                    
def getRoutes(route_url,routetype):
    print "route start"
    #route_url = "http://www.ctsfj.com/line/show.asp?id=597"
    #此处开始获取线路信息
    html = urllib2.urlopen(route_url).read()
    html = getUTF(html)
    #
    #print html
    route_titles = route_title_re.findall(html)
    if route_titles:
        for route_title,route_price in route_titles:
            print route_title
            route_price = route_price.replace("￥","").replace("起","")
            #print route_price
    #
    route_detail = route_detail_re.findall(html)
    if route_detail:
        route_detail = route_detail[0].strip()
        #print route_detail
        
    #
    route_plane = ""
    route_days = ""
    route_sights = ""
    place_list = route_planes_re.findall(html)
    if place_list:
        route_days = len(place_list)
        daynum =1
        for sights, daydetail in place_list:
            route_plane = route_plane + "#第" + str(daynum) + "天#"
            route_plane = route_plane + strip_tags(daydetail).strip()
            rmsight = "第"+str(daynum)+"天"
            sights = sights.strip().replace(rmsight,"").replace("-",",").replace("北京","").replace(" ","")+","
            sightss = sights.split(",")
            for k in sightss:
                if route_sights.find(k)<0:
                    route_sights = route_sights +sights
            daynum = daynum+1
    #print route_plane
    if route_days=="":
        route_days = 1
    #print route_days
    #print route_sights

    #
    depcity = "北京"
    #
    route_dates=""
    route_date = route_dates_re.findall(html)
    for i in route_date:
        route_dates = route_dates+i+";"
    #print route_dates
    #
    route_hotel = ""
    route_meal = ""
    route_traffic01 = ""
    route_tickets = ""
    route_payown = ""
    route_trip = route_trip_all_re.findall(html)
    if route_trip:
        for route_trips in route_trip[0].split("<br />"):
            if route_trips.find("住宿")>-1:
                if route_trips.find("：")>-1:
                    route_trips = (route_trips.split("："))[1]
                route_hotel = re_h.sub('',route_trips)
            if route_trips.find("用餐")>-1:
                if route_trips.find("：")>-1:
                    route_trips = (route_trips.split("："))[1]
                route_meal = re_h.sub('',route_trips)
            if route_trips.find("交通")>-1:
                if route_trips.find("：")>-1:
                    route_trips = (route_trips.split("："))[1]
                route_traffic01 = re_h.sub('',route_trips)
            if route_trips.find("门票")>-1:
                if route_trips.find("：")>-1:
                    route_trips = (route_trips.split("："))[1]
                route_tickets = re_h.sub('',route_trips)
        route_payown = route_trip[1].strip()
        #print route_payown
    #
    route_go_back = getTrafficType(route_title)
    #print route_go_back
    #入库
    route = Route()
    route.site = '40'
    route.supplier_url = "http://www.cytsbj.com/"
    route.supplier = "北京青年旅行社"
    route.title = route_title
    route.rout_url = route_url
    route.outcity = depcity
    route.sights = route_sights
    route.telphone = route_telphone
    route.detail = route_detail
    route.dest = route_sights
    route.guide = ''
    route.type = routetype
    route.dates = route_dates
    route.price = int(route_price)
    route.go_t = route.back_t = route_go_back
    route.plane = route_plane
    route.days = route_days

    route.hotel = route_hotel
    route.meal = route_meal
    route.traffic01 = route_traffic01
    route.tickets = route_tickets
    route.payown = route_payown
    
    getschedules = getSchedules(html)
    if getschedules=='no':
        print "==no schedule=="
    else:
        route.schedules = getschedules
        route.updRoute_Schedules()
    route.updRoute()
 
    #print "++++++"
    
Schedules_re = re.compile(r'''<div\s*class=.hd3.>\s*<h3>([\S\s]+?)</h3>\s*</div>\s*<div\s*class=.bd3.>([\S\s]+?)</div>''', re.X|re.S)
Schedules_brief_re = re.compile(r'''([\S\s]+?)<span>''', re.X|re.S)
Schedules_place_re = re.compile(r'''''', re.X|re.S)
Schedules_eat_tra_hot_re = re.compile(r'''(<span>[\S\s]+)''', re.X|re.S)


def getSchedules(html):
     
     schedules_list = Schedules_re.findall(html)
     schedules = range(0,len(schedules_list))
     n = 0
     for place_info, plane_info in schedules_list:
        n+=1
        schedule = Schedule()
        s_brief = Schedules_brief_re.findall(plane_info)
        if s_brief:
            schedule.s_brief = strip_tags(s_brief[0]).strip()
        #print "s:"+strip_tags(s_brief[0]).strip()
        
        rmplace = "第"+str(n)+"天"
        s_place = place_info.strip().replace(rmplace,"").replace("-",",").replace(" ","")
        schedule.s_place= s_place
        #print "s:"+s_place
        
        schedule.s_traffic = ""
        
        #print plane_info
        schedule_hbld = Schedules_eat_tra_hot_re.findall(plane_info)
        if schedule_hbld:
            schedule_hbld = strip_tags(schedule_hbld[0]).strip().replace(" ","")
            #print schedule_hbld
        sttr = schedule_hbld.split("：")
        if len(sttr)==5:
            schedule.s_hotel = sttr[4]
            #print sttr[4]
            #print sttr[1]
            #print sttr[2]
            #print sttr[3]
            schedule.s_brf="1" if sttr[1].find('有')>-1 else "0"
            schedule.s_lunch="1" if sttr[2].find('有')>-1 else "0"
            schedule.s_dinner="1" if sttr[3].find('有')>-1 else "0"
        
        schedule.s_num = n
        #print "serialnumber:"+str(n)
        schedules[n-1]=schedule
     return schedules


def getPlane(html):
     return plane_str

def getTrafficType(t):
    t1='-'
    if t.find('汽')>-1:
        t1='汽车'
    if t.find('动')>-1:
        t1='动车'
    if t.find('卧')>-1:
        t1='卧铺'
    if t.find('火车')>-1:
        t1='火车'
    if t.find('船')>-1:
        t1='邮轮'
    if t.find('邮')>-1:
        t1='邮轮'
    if t.find('飞')>-1:
        t1='飞机'
    if t.find('机')>-1:
        t1='飞机'
    return t1

#去掉文本标签
def strip_tags(html):
    from HTMLParser import HTMLParser
    html=html.strip()
    html=html.strip("\n")
    result=[]
    parse=HTMLParser()
    parse.handle_data=result.append
    parse.feed(html)
    parse.close()
    return "".join(result)

def getUTF(gb):
    #uni=unicode(gb,'gb2312')
    utf = gb.decode('gbk','ignore').encode('utf-8')
    return utf

if __name__ == '__main__':
    script_path = os.path.dirname(os.path.realpath(__file__))
    os.chdir(script_path)
    do_debug = False

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], 'hdno:p:', ['help', 'debug', 'no-cache', 'only-cache', 'provider'])
    except getopt.GetoptError:
        usage()
        sys.exit()

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-d', '--debug'):
            do_debug = True
        elif opt in ('-n', '--no-cache'):
            use_cache = False
        elif opt in ('-o', '--only-cache'):
            if arg.lower() in ('no', 'n', '0'):
                only_cache = False
            else:
                only_cache = True
        elif opt in ('-p', '--provider'):
            pass

    if do_debug:
        import pdb
        pdb.set_trace()
    do_crawl()
